Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): yield inside semantic analysis and large projection passes (#1542)

Kaoru0822ca7ef69c0cee
diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts
index 3fac39dbb..72fb67edd 100644
--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts
+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts
@@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises";
import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";
import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";
import {
- analyzeParsedJavaScriptSemantics,
+ analyzeParsedJavaScriptSemanticsSteps,
classifyParsedJavaScriptOpenReceivers,
} from "../../domain/javascript/javascriptSemanticAnalysis.js";
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
@@ -189,6 +189,7 @@ function* analyzeArtifactFileSteps<
return;
}
const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);
+ yield;
const analysis = analyzeParsedJavaScriptStaticSource(
file.text.value,
parsed,
@@ -198,7 +199,7 @@ function* analyzeArtifactFileSteps<
yield;
const semantics =
analysis.parse_status === "complete" || analysis.parse_status === "partial"
- ? analyzeParsedJavaScriptSemantics(parsed)
+ ? yield* analyzeParsedJavaScriptSemanticsSteps(parsed)
: null;
const projected =
semantics === null
diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts
index 770930de7..a0cc8ccf8 100644
--- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts
+++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts
@@ -287,38 +287,50 @@ function* projectFileSteps(
});
if (moduleNode === null) return [];
state.roots.add(moduleNode.node_id);
- const callableNodes = new Map(
- ir.callables.flatMap((callable) => {
- const node = retainNode(state, file, {
- kind: "function",
- roleKey: `callable:${callable.callableId}`,
- location: callable.location,
- label: callable.name,
- functionNodeId: null,
- // The label is display text; keep the exact name, which may be "".
- properties: { name: callable.name },
- });
- return node === null ? [] : [[callable.callableId, node] as const];
- }),
- );
+ // Node creation keeps source order, which the per-file node budget relies
+ // on; yields between nodes do not change it.
+ const callableNodes = new Map<string, JavaScriptSemanticGraphNode>();
+ for (const [index, callable] of ir.callables.entries()) {
+ if (index % 64 === 0) yield;
+ const node = retainNode(state, file, {
+ kind: "function",
+ roleKey: `callable:${callable.callableId}`,
+ location: callable.location,
+ label: callable.name,
+ functionNodeId: null,
+ // The label is display text; keep the exact name, which may be "".
+ properties: { name: callable.name },
+ });
+ if (node !== null) callableNodes.set(callable.callableId, node);
+ }
const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes);
- const bindingNodes = new Map(
- ir.bindings.flatMap((binding) => {
- const location = binding.definitions[0]?.location ?? null;
- const kind = binding.kind === "parameter" ? "parameter" : "binding";
- const owner = callableOwnerAt(location);
- const node = retainNode(state, file, {
- kind,
- roleKey: `binding:${binding.bindingId}`,
- location,
- label: binding.name,
- functionNodeId: owner?.node_id ?? null,
- });
- return node === null ? [] : [[binding.bindingId, node] as const];
- }),
+ const bindingNodes = new Map<string, JavaScriptSemanticGraphNode>();
+ for (const [index, binding] of ir.bindings.entries()) {
+ if (index % 64 === 0) yield;
+ const location = binding.definitions[0]?.location ?? null;
+ const kind = binding.kind === "parameter" ? "parameter" : "binding";
+ const owner = callableOwnerAt(location);
+ const node = retainNode(state, file, {
+ kind,
+ roleKey: `binding:${binding.bindingId}`,
+ location,
+ label: binding.name,
+ functionNodeId: owner?.node_id ?? null,
+ });
+ if (node !== null) bindingNodes.set(binding.bindingId, node);
+ }
+ const returnSiteNodes = yield* createReturnSiteNodes(
+ file,
+ ir,
+ callableNodes,
+ state,
+ );
+ const callSiteNodes = yield* createCallSiteNodes(
+ file,
+ ir,
+ callableNodes,
+ state,
);
- const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state);
- const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state);
const definitions: Omit<FileContext, "referenceNodesWithin"> = {
file,
ir,
@@ -339,8 +351,7 @@ function* projectFileSteps(
callableOwnerAt,
callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),
};
- yield;
- projectDefinitionsAndReferences(definitions);
+ yield* projectDefinitionsAndReferences(definitions);
const context: FileContext = {
...definitions,
referenceNodesWithin: createSemanticNodeRangeLookup(
@@ -351,7 +362,8 @@ function* projectFileSteps(
// keeps projection order and results unchanged.
for (const project of FILE_PROJECTION_PASSES) {
yield;
- project(context);
+ const steps = project(context);
+ if (steps !== undefined) while (steps.next().done !== true) yield;
}
yield;
return projectSemanticFunctionFingerprints(
@@ -362,14 +374,15 @@ function* projectFileSteps(
);
}
-const createReturnSiteNodes = (
+function* createReturnSiteNodes(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
state: BuilderState,
-): Map<string, JavaScriptSemanticGraphNode> => {
+): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
const result = new Map<string, JavaScriptSemanticGraphNode>();
- for (const callable of ir.callables) {
+ for (const [index, callable] of ir.callables.entries()) {
+ if (index % 64 === 0) yield;
const owner = callables.get(callable.callableId);
if (owner === undefined) continue;
for (const site of callable.returnSites) {
@@ -385,16 +398,17 @@ const createReturnSiteNodes = (
}
}
return result;
-};
+}
-const createCallSiteNodes = (
+function* createCallSiteNodes(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
state: BuilderState,
-): Map<string, JavaScriptSemanticGraphNode> => {
+): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
const result = new Map<string, JavaScriptSemanticGraphNode>();
- for (const call of ir.callSites) {
+ for (const [index, call] of ir.callSites.entries()) {
+ if (index % 64 === 0) yield;
const owner =
call.callerCallableId === null
? null
@@ -409,12 +423,13 @@ const createCallSiteNodes = (
if (node !== null) result.set(call.callSiteId, node);
}
return result;
-};
+}
-const projectDefinitionsAndReferences = (
+function* projectDefinitionsAndReferences(
context: Omit<FileContext, "referenceNodesWithin">,
-): void => {
- for (const binding of context.ir.bindings) {
+): Generator<void, void> {
+ for (const [bindingIndex, binding] of context.ir.bindings.entries()) {
+ if (bindingIndex % 64 === 0) yield;
const bindingNode = context.bindingNodes.get(binding.bindingId);
if (bindingNode === undefined) continue;
for (const [index, definition] of binding.definitions.entries()) {
@@ -434,6 +449,7 @@ const projectDefinitionsAndReferences = (
}
}
for (const [index, reference] of context.ir.references.entries()) {
+ if (index % 64 === 0) yield;
const expression = retainNode(context.state, context.file, {
kind: "expression",
roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`,
@@ -464,7 +480,7 @@ const projectDefinitionsAndReferences = (
resolution: "resolved",
});
}
-};
+}
const projectCalls = (context: FileContext): void => {
for (const call of context.ir.callSites) {
@@ -551,7 +567,9 @@ const createArgumentNode = (
});
/** Per-file projection passes, run in this order with a yield between each. */
-const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [
+const FILE_PROJECTION_PASSES: readonly ((
+ context: FileContext,
+) => Iterator<void, void> | void)[] = [
projectSemanticValues,
projectSemanticObjects,
projectCalls,
diff --git a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts
index be801ab86..e98526160 100644
--- a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts
+++ b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts
@@ -240,10 +240,11 @@ export const projectSemanticClosureCaptures = (
};
/** Retain bounded unresolved dynamic-call, dynamic-property and dynamic-scope frontiers. */
-export const projectSemanticFrontiers = (
+export function* projectSemanticFrontiers(
context: SemanticFlowProjectionContext,
-): void => {
- for (const frontier of context.ir.frontiers) {
+): Generator<void, void> {
+ for (const [index, frontier] of context.ir.frontiers.entries()) {
+ if (index % 64 === 0) yield;
// Dynamic environments leave identifier reads and writes unresolved.
const family =
frontier.kind === "dynamic-call"
@@ -275,4 +276,4 @@ export const projectSemanticFrontiers = (
);
addSemanticGraphUnknown(context.state, unknown);
}
-};
+}
diff --git a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts
index 1b71b2784..337899f83 100644
--- a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts
+++ b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts
@@ -20,13 +20,14 @@ import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlo
import { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js";
/** Project static object reads, writes, spreads, and destructuring. */
-export const projectSemanticObjects = (
+export function* projectSemanticObjects(
context: SemanticFlowProjectionContext,
-): void => {
+): Generator<void, void> {
const values = new Map(
context.ir.bindings.map((binding) => [binding.bindingId, binding.value]),
);
- for (const operation of context.ir.objectOperations) {
+ for (const [index, operation] of context.ir.objectOperations.entries()) {
+ if (index % 64 === 0) yield;
const occurrence = addObjectOccurrence(context, operation);
const target = objectTarget(context, values, operation);
const ends = objectRelationEnds(context, operation, occurrence, target);
@@ -43,7 +44,7 @@ export const projectSemanticObjects = (
if (!resolved)
addObjectUnknown(context, operation, target.slot ?? occurrence, target);
}
-};
+}
const OBJECT_RELATIONS = {
read: "reads-property",
diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts
index 4b5ad0f72..50af0f2cb 100644
--- a/src/domain/javascript/javascriptSemanticAnalysis.ts
+++ b/src/domain/javascript/javascriptSemanticAnalysis.ts
@@ -38,7 +38,7 @@ import {
collectSemanticReturns,
resolveSemanticModuleCallables,
} from "./javascriptSemanticReturns.js";
-import { collectJavaScriptDerivedSemantics } from "./javascriptSemanticDerivedAnalysis.js";
+import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js";
import { range } from "./javascriptStaticAnalysisHelpers.js";
import { propertyName } from "./javascriptAstValues.js";
import { semanticCoverage } from "./javascriptSemanticCoverage.js";
@@ -107,18 +107,40 @@ export const analyzeParsedJavaScriptReferences = (
export const analyzeParsedJavaScriptSemantics = (
file: ParsedJavaScriptSource,
): JavaScriptSemanticIr => {
+ const steps = analyzeParsedJavaScriptSemanticsSteps(file);
+ for (;;) {
+ const step = steps.next();
+ if (step.done === true) return step.value;
+ }
+};
+
+/**
+ * Recover semantics from a parsed artifact with a yield between analysis
+ * phases, so a caller can serve control messages within one large file.
+ * Each phase completes before the next starts; results equal the synchronous
+ * form.
+ */
+export function* analyzeParsedJavaScriptSemanticsSteps(
+ file: ParsedJavaScriptSource,
+): Generator<void, JavaScriptSemanticIr> {
const state = createState(file.program);
collectDefinitions(file.program, state);
+ yield;
collectSemanticMemberMutations(file.program, state);
+ yield;
traverseJavaScriptAst(file.program, {
enter: (node) => collectSemanticModuleLink(node, state),
});
+ yield;
const references = collectSemanticReferences(file.program, state);
+ yield;
const parserPartial = file.errors.length > 0;
const bindings = immutableSemanticBindings(state);
+ yield;
const callables = collectSemanticReturns(file.program, state, parserPartial);
+ yield;
const moduleLinks = resolveSemanticModuleCallables(state, callables);
- const derived = collectJavaScriptDerivedSemantics(
+ const derived = yield* collectJavaScriptDerivedSemanticsSteps(
file.program,
state,
callables,
@@ -154,7 +176,7 @@ export const analyzeParsedJavaScriptSemantics = (
"Cross-function mutation and dynamic property resolution remain unknown.",
],
};
-};
+}
/** Receiver identity facts for the overloaded `.open` syntax only. */
export type JavaScriptOpenReceiverFact =
diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts
new file mode 100644
index 000000000..9aaa71d93
--- /dev/null
+++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts
@@ -0,0 +1,34 @@
+import { expect, it } from "vitest";
+
+import {
+ analyzeParsedJavaScriptSemantics,
+ analyzeParsedJavaScriptSemanticsSteps,
+} from "./javascriptSemanticAnalysis.js";
+import { parseJavaScriptSource } from "./javascriptSourceParser.js";
+
+const source = `
+import { readFile } from "node:fs";
+const shared = { token: "TOKEN" };
+export function load(path) {
+ return new Promise((resolve) => setTimeout(() => resolve(readFile(path)), 1));
+}
+export const box = { get() { return shared; } };
+box.get().value = process.env.MODE ?? "default";
+`;
+
+it("yields between semantic phases and returns the synchronous result (#1462)", () => {
+ const parsed = parseJavaScriptSource(source, "main.js");
+ if (parsed === null) throw new Error("Expected parsed source");
+ const steps = analyzeParsedJavaScriptSemanticsSteps(parsed);
+ let yields = 0;
+ let step = steps.next();
+ while (step.done !== true) {
+ yields += 1;
+ step = steps.next();
+ }
+ // Every top-level phase and derived collector is a separate step.
+ expect(yields).toBeGreaterThanOrEqual(13);
+ const reparsed = parseJavaScriptSource(source, "main.js");
+ if (reparsed === null) throw new Error("Expected parsed source");
+ expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));
+});
diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts
index d3e630ac9..545652a74 100644
--- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts
+++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts
@@ -36,43 +36,54 @@ type DerivedSemanticAnalysis = Pick<
>;
/** Compose bounded semantic passes after lexical definitions are available. */
-export const collectJavaScriptDerivedSemantics = (
+/**
+ * Collect derived semantics with a yield between collectors. Each collector
+ * completes before the next starts, so results equal the synchronous form.
+ */
+export function* collectJavaScriptDerivedSemanticsSteps(
program: t.Program,
state: JavaScriptSemanticAnalysisState,
callables: readonly JavaScriptSemanticCallable[],
parserPartial: boolean,
-): DerivedSemanticAnalysis => {
+): Generator<void, DerivedSemanticAnalysis> {
const calls = collectJavaScriptSemanticCalls(program, state, callables);
+ yield;
const promiseOperations = collectJavaScriptSemanticPromises(
program,
state,
callables,
);
+ yield;
const asyncEffects = collectJavaScriptSemanticAsyncEffects(
program,
state,
callables,
);
+ yield;
const childProcesses = collectJavaScriptSemanticChildProcesses(
program,
state,
callables,
);
+ yield;
const dataEffects = collectJavaScriptSemanticDataEffects(
program,
state,
callables,
);
+ yield;
const resourceOperations = collectJavaScriptSemanticResources(
program,
state,
callables,
);
+ yield;
const objectOperations = collectJavaScriptSemanticObjects(
program,
state,
callables,
);
+ yield;
const functionFingerprints = collectJavaScriptSemanticFingerprints({
state,
callables,
@@ -96,4 +107,4 @@ export const collectJavaScriptDerivedSemantics = (
objectOperations,
functionFingerprints,
};
-};
+}