Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): yield during per-file projection and graph construction (#1525)
Kaoru08225fdb0636b473
diff --git a/src/application/javascript/JavaScriptApplicationCancellation.test.ts b/src/application/javascript/JavaScriptApplicationCancellation.test.tsindex 1457046ef..5c2b2f005 100644--- a/src/application/javascript/JavaScriptApplicationCancellation.test.ts+++ b/src/application/javascript/JavaScriptApplicationCancellation.test.ts@@ -42,6 +42,46 @@ describe("JavaScript analysis cancellation before publication", () => {expect(phases).not.toContain("validate_javascript_application_result");});+ it("handles control turns and cancellation inside one large source file (#1462)", async () => {+ const root = await createTestTempDirectory("rea-js-cancel-in-file-");+ await writeFile(+ join(root, "index.js"),+ Array.from(+ { length: 1200 },+ (_, index) =>+ `export function f${index}(){return {name:'f${index}',value:${index}};}`,+ ).join("\n"),+ );+ const controller = new AbortController();+ const phases: string[] = [];+ let turns = 0;+ // Count event-loop turns once the file starts; parsing, analysis and+ // projection of this one file must leave room for them.+ const tick = (): void => {+ turns += 1;+ if (turns === 3) controller.abort();+ else if (!controller.signal.aborted) setImmediate(tick);+ };+ const result = await analyzeJavaScriptApplication(+ { input_path: root, format: "directory" },+ {+ signal: controller.signal,+ progress: {+ async report(event) {+ phases.push(event.phase);+ if (event.phase === "parse_javascript_source") setImmediate(tick);+ },+ },+ },+ );+ if (result.ok) throw new Error("Cancelled analysis must not publish");+ expect(projectAnalysisError(result.error)).toMatchObject({+ code: "cancelled",+ });+ expect(phases).toContain("parse_javascript_source");+ expect(phases).not.toContain("build_javascript_application_graph");+ });+it.each(["parse_javascript_source","build_javascript_application_graph",diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.tsindex 84390b873..3fac39dbb 100644--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts@@ -33,6 +33,7 @@ import type {JavaScriptSourceMapOriginal,} from "./JavaScriptArtifactAnalysisTypes.js";import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js";+import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js";import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js";interface MutableArtifactAnalysis<@@ -56,34 +57,44 @@ interface ArtifactAnalysisContext<readonly projectSemantics: (file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,- ) => SemanticIr;+ ) => Generator<void, SemanticIr>;}/** Analyze every text source from the artifact without a result quota. */export const analyzeJavaScriptArtifactFiles = (fileSet: JavaScriptArtifactFileSet,): JavaScriptArtifactAnalysis =>- analyzeArtifactFiles(fileSet, (_file, ir) => ir);+ analyzeArtifactFiles(fileSet, function* (_file, ir) {+ // The full IR needs no projection; one step keeps the shared protocol.+ yield;+ return ir;+ });-/** Project each file's full IR immediately, retaining only later module facts. */+/**+ * Project each file's full IR immediately, retaining only later module facts.+ * Parsing, analysis and each projection pass run as separate steps, so control+ * messages and cancellation are handled within one large file as well.+ */export const analyzeAndProjectJavaScriptArtifactFiles = async (fileSet: JavaScriptArtifactFileSet,projectSemantics: (file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,- ) => void,+ ) => Iterator<void, void> | void,beforeFile?: (file: JavaScriptArtifactFile,completed: number,total: number,) => Promise<void>,+ signal?: AbortSignal,): Promise<JavaScriptModuleArtifactAnalysis> => {const state = emptyArtifactAnalysis<JavaScriptModuleSemanticIr>();- const projectFileSemantics = (+ const projectFileSemantics = function* (file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,- ): JavaScriptModuleSemanticIr => {- projectSemantics(file, ir);+ ): Generator<void, JavaScriptModuleSemanticIr> {+ const projection = projectSemantics(file, ir);+ if (projection !== undefined) while (projection.next().done !== true) yield;const programScopes = ir.scopes.filter(({ kind }) => kind === "program");const programScopeId = programScopes[0]?.scopeId;const localNames = new Set(@@ -111,7 +122,10 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (// Let progress, cancellation and garbage collection run between files.await setImmediate();await beforeFile?.(file, index, fileSet.files.length);- analyzeArtifactFile(file, context);+ await completeJavaScriptAnalysisSteps(+ analyzeArtifactFileSteps(file, context),+ signal,+ );}return finalizeArtifactAnalysis(state);};@@ -122,7 +136,10 @@ const analyzeArtifactFiles = <SemanticIr extends JavaScriptModuleSemanticIr>(): JavaScriptArtifactAnalysis<SemanticIr> => {const state = emptyArtifactAnalysis<SemanticIr>();const context = { state, projectSemantics };- for (const file of fileSet.files) analyzeArtifactFile(file, context);+ for (const file of fileSet.files) {+ const steps = analyzeArtifactFileSteps(file, context);+ while (steps.next().done !== true);+ }return finalizeArtifactAnalysis(state);};@@ -148,10 +165,12 @@ const finalizeArtifactAnalysis = <};};-const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(+function* analyzeArtifactFileSteps<+ SemanticIr extends JavaScriptModuleSemanticIr,+>(file: JavaScriptArtifactFile,context: ArtifactAnalysisContext<SemanticIr>,-): void => {+): Generator<void, void> {const { state } = context;addStructuredObservations(file, state);if (file.kind === "html" && file.text.included)@@ -162,6 +181,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(return;}const parsed = parseJavaScriptSource(file.text.value, file.path);+ yield;if (parsed === null) {const analysis = failedJavaScriptStaticAnalysis();state.files.push({ file, javascript: analysis, semantic: null });@@ -175,17 +195,19 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(openReceiverFacts,);const staticFindings = findingCount(analysis);+ yield;const semantics =analysis.parse_status === "complete" || analysis.parse_status === "partial"? analyzeParsedJavaScriptSemantics(parsed): null;+ const projected =+ semantics === null+ ? null+ : yield* context.projectSemantics(file, semantics);state.files.push({file,javascript: analysis,- semantic:- semantics === null- ? null- : { ir: context.projectSemantics(file, semantics) },+ semantic: projected === null ? null : { ir: projected },});state.visitedNodes += analysis.visited_ast_nodes;state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0);@@ -194,7 +216,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(0,);if (analysis.parse_status === "failed") state.parseFailures += 1;-};+}const addSourceMap = (file: JavaScriptArtifactFile,diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.tsindex bf46a0557..023802a79 100644--- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts+++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts@@ -1,7 +1,7 @@import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";import {createJavaScriptApplicationGraph,- createImmutableJavaScriptApplicationGraphSteps,+ sealTransferredJavaScriptApplicationGraphSteps,type JavaScriptApplicationGraph,} from "../../domain/javascript/javascriptApplicationGraph.js";import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js";@@ -15,9 +15,9 @@ import {addJavaScriptHtmlRoles,addJavaScriptSourceMapOriginals,} from "./JavaScriptArtifactGraphDocuments.js";-import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js";+import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js";import {- addJavaScriptModuleRelationships,+ addJavaScriptModuleRelationshipsSteps,type JavaScriptModuleRelationshipOmissions,addJavaScriptSourceModules,} from "./JavaScriptModuleRelationships.js";@@ -51,24 +51,42 @@ export const buildJavaScriptArtifactGraph = (analysis: JavaScriptModuleArtifactAnalysis,): JavaScriptApplicationGraph =>createJavaScriptApplicationGraph(- buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),+ completeSynchronously(+ buildJavaScriptArtifactGraphInputSteps(snapshot, fileSet, analysis),+ ),);-/** Build a validated application graph with cooperative immutable ownership. */-export const buildImmutableJavaScriptArtifactGraphSteps = (+/**+ * Build a validated application graph with cooperative immutable ownership.+ * Constructing the graph input also runs in steps: projecting every module+ * relationship and finding at once held the event loop for seconds.+ */+export function* buildImmutableJavaScriptArtifactGraphSteps(snapshot: ArtifactInventorySnapshot,fileSet: JavaScriptArtifactFileSet,analysis: JavaScriptModuleArtifactAnalysis,-): Generator<void, JavaScriptApplicationGraph> =>- createImmutableJavaScriptApplicationGraphSteps(- buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),+): Generator<void, JavaScriptApplicationGraph> {+ const input = yield* buildJavaScriptArtifactGraphInputSteps(+ snapshot,+ fileSet,+ analysis,);+ // The input was built here and is not retained, so it can be transferred.+ return yield* sealTransferredJavaScriptApplicationGraphSteps(input);+}-const buildJavaScriptArtifactGraphInput = (+const completeSynchronously = <Value>(steps: Iterator<void, Value>): Value => {+ for (;;) {+ const step = steps.next();+ if (step.done === true) return step.value;+ }+};++function* buildJavaScriptArtifactGraphInputSteps(snapshot: ArtifactInventorySnapshot,fileSet: JavaScriptArtifactFileSet,analysis: JavaScriptModuleArtifactAnalysis,-): unknown => {+): Generator<void, unknown> {const accumulator = new JavaScriptArtifactGraphAccumulator();const root = createJavaScriptArtifactRootNode(accumulator, snapshot);const context: JavaScriptArtifactGraphContext = {@@ -86,15 +104,23 @@ const buildJavaScriptArtifactGraphInput = (containerNodes: new Map([[snapshot.manifest.root_sha256, root]]),};addJavaScriptArtifactContainers(context);+ yield;addJavaScriptArtifactFiles(context);+ yield;const packageRoots = addJavaScriptPackageNodes(context);+ yield;addJavaScriptSourceModules(context);+ yield;const bundlerLimitations = addJavaScriptBundlerNodes(context);- const relationshipOmissions = addJavaScriptModuleRelationships(context);- const findingLimitations = addJavaScriptStaticFindings(context);+ const relationshipOmissions =+ yield* addJavaScriptModuleRelationshipsSteps(context);+ const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context);addElectronBoundaries(context);+ yield;addJavaScriptHtmlRoles(context);+ yield;addJavaScriptSourceMapOriginals(context);+ yield;const coverage = graphCoverage(context);return {schema: "JavaScriptApplicationGraph",@@ -111,7 +137,7 @@ const buildJavaScriptArtifactGraphInput = (...graphLimitations(context, coverage.status, relationshipOmissions),],};-};+}const graphCoverage = (context: JavaScriptArtifactGraphContext) => {const resourceLimits = semanticResourceLimits(context);diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.tsindex 41f480daf..e7a002a6a 100644--- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts+++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts@@ -44,11 +44,12 @@ interface ResolvedReference {}/** Project imports, workers, roles, endpoints, storage, and source-map links. */-export const addJavaScriptStaticFindings = (+export function* addJavaScriptStaticFindingsSteps(context: JavaScriptArtifactGraphContext,-): readonly string[] => {+): Generator<void, readonly string[]> {const omissions = { selfReferences: 0 };for (const analyzed of context.analysis.files) {+ yield;const { file, javascript } = analyzed;const asset = context.assetNodes.get(file.path);if (javascript === null || asset === undefined) continue;@@ -69,7 +70,7 @@ export const addJavaScriptStaticFindings = (["static reference", "static references"],"the referencing module",);-};+}const addReference = (context: JavaScriptArtifactGraphContext,diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.tsindex ab986608a..25919923c 100644--- a/src/application/javascript/JavaScriptArtifactReconstruction.ts+++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts@@ -90,7 +90,7 @@ export const reconstructJavaScriptArtifact = async (const semanticProjection = createJavaScriptSemanticGraphProjection();const analysis = await analyzeAndProjectJavaScriptArtifactFiles(files,- semanticProjection.projectFile,+ semanticProjection.projectFileSteps,async (file, completed, total) => {abortIfNeeded(signal);await progress?.report({@@ -101,6 +101,7 @@ export const reconstructJavaScriptArtifact = async (});abortIfNeeded(signal);},+ signal,);await checkpointJavaScriptAnalysis(signal);abortIfNeeded(signal);diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.tsindex 8e12f3beb..3784141a8 100644--- a/src/application/javascript/JavaScriptModuleRelationships.ts+++ b/src/application/javascript/JavaScriptModuleRelationships.ts@@ -136,15 +136,17 @@ export interface JavaScriptModuleRelationshipOmissions {type RelationshipOmissionCounter = { selfImports: number };/** Compose bounded CommonJS and ESM binding relationships across artifact files. */-export const addJavaScriptModuleRelationships = (+export function* addJavaScriptModuleRelationshipsSteps(context: JavaScriptArtifactGraphContext,-): JavaScriptModuleRelationshipOmissions => {+): Generator<void, JavaScriptModuleRelationshipOmissions> {const omissions = { selfImports: 0 };for (const analyzed of context.analysis.files) {const { file, semantic } = analyzed;const source = context.sourceModuleNodes.get(file.path);if (semantic === null || source === undefined) continue;for (const link of semantic.ir.moduleLinks) {+ // One module can export thousands of bindings, each with return shapes.+ yield;const input = { context, file, semantic, source, link };if (isExportLink(link)) addExportRelationship(input, omissions);else if (link.specifier !== null)@@ -155,7 +157,7 @@ export const addJavaScriptModuleRelationships = (}}return omissions;-};+}const addExportRelationship = (input: RelationshipInput,diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.tsindex 45ded5051..770930de7 100644--- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts+++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts@@ -1,6 +1,6 @@import {createJavaScriptSemanticGraph,- createImmutableJavaScriptSemanticGraphSteps,+ sealTransferredJavaScriptSemanticGraphSteps,type JavaScriptSemanticGraph,} from "../../domain/javascript/javascriptSemanticGraph.js";import type {@@ -130,6 +130,11 @@ export interface JavaScriptSemanticGraphProjection {file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,) => void;+ /** Project one file in steps between its projection passes. */+ readonly projectFileSteps: (+ file: JavaScriptArtifactFile,+ ir: JavaScriptSemanticIr,+ ) => Generator<void, void>;readonly finish: (rootArtifactSha256: string,applicationGraph: BuilderInput["applicationGraph"],@@ -146,10 +151,10 @@ export const createJavaScriptSemanticGraphProjection =const state = emptyState({ nodes: [] });const fingerprints: JavaScriptSemanticFingerprint[] = [];let truncatedFiles = 0;- const projectSource = (+ const projectSourceSteps = function* (file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,- ): void => {+ ): Generator<void, void> {if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) {truncatedFiles += 1;return;@@ -161,7 +166,7 @@ export const createJavaScriptSemanticGraphProjection =remainingTreeBudget,);state.fileNodesDropped = false;- fingerprints.push(...projectFile(file, ir, state));+ fingerprints.push(...(yield* projectFileSteps(file, ir, state)));// Nodes were dropped only if the budget actually blocked creation. A file// that exactly fills its share ends with a zero budget without dropping// anything, so the remaining budget alone cannot decide truncation.@@ -198,7 +203,8 @@ export const createJavaScriptSemanticGraphProjection =evidence_contexts: state.evidenceContexts.contexts,nodes: [...state.nodes.values()],relations,- fingerprints,+ // The input is transferred to the factory; `fingerprints` is cleared below.+ fingerprints: [...fingerprints],unknowns,coverage: {status: truncatedFiles > 0 ? "partial" : "unknown",@@ -246,7 +252,11 @@ export const createJavaScriptSemanticGraphProjection =return graph;};return {- projectFile: projectSource,+ projectFile: (file, ir) => {+ const steps = projectSourceSteps(file, ir);+ while (steps.next().done !== true);+ },+ projectFileSteps: projectSourceSteps,finish: (rootArtifactSha256, applicationGraph) =>finishWith(createJavaScriptSemanticGraph,@@ -255,18 +265,19 @@ export const createJavaScriptSemanticGraphProjection =),finishImmutableSteps: (rootArtifactSha256, applicationGraph) =>finishWith(- createImmutableJavaScriptSemanticGraphSteps,+ sealTransferredJavaScriptSemanticGraphSteps,rootArtifactSha256,applicationGraph,),};};-const projectFile = (+function* projectFileSteps(file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,state: BuilderState,-): JavaScriptSemanticFingerprint[] => {+): Generator<void, JavaScriptSemanticFingerprint[]> {+ yield;const moduleNode = retainNode(state, file, {kind: "module",roleKey: "module",@@ -328,6 +339,7 @@ const projectFile = (callableOwnerAt,callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),};+ yield;projectDefinitionsAndReferences(definitions);const context: FileContext = {...definitions,@@ -335,26 +347,20 @@ const projectFile = (definitions.referenceNodes,),};- projectSemanticValues(context);- projectSemanticObjects(context);- projectCalls(context);- projectSemanticPromises(context);- projectSemanticEvents(context);- projectSemanticTimers(context);- projectSemanticChildProcesses(context);- projectSemanticConfiguration(context);- projectSemanticRequests(context);- projectSemanticBoundaries(context);- projectSemanticResources(context);- projectSemanticClosureCaptures(context);- projectSemanticFrontiers(context);+ // Each pass completes before the next starts, so yielding between passes+ // keeps projection order and results unchanged.+ for (const project of FILE_PROJECTION_PASSES) {+ yield;+ project(context);+ }+ yield;return projectSemanticFunctionFingerprints(file,ir,callableNodes,state.evidenceContexts,);-};+}const createReturnSiteNodes = (file: JavaScriptArtifactFile,@@ -543,3 +549,20 @@ const createArgumentNode = (: `argument ${String(argument.index)}`,functionNodeId: callNode.function_node_id,});++/** Per-file projection passes, run in this order with a yield between each. */+const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [+ projectSemanticValues,+ projectSemanticObjects,+ projectCalls,+ projectSemanticPromises,+ projectSemanticEvents,+ projectSemanticTimers,+ projectSemanticChildProcesses,+ projectSemanticConfiguration,+ projectSemanticRequests,+ projectSemanticBoundaries,+ projectSemanticResources,+ projectSemanticClosureCaptures,+ projectSemanticFrontiers,+];diff --git a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.tsindex c96ccece7..a460de077 100644--- a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts+++ b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts@@ -11,11 +11,13 @@ import {createJavaScriptApplicationGraph,createImmutableJavaScriptApplicationGraphSteps,isValidatedImmutableJavaScriptApplicationGraph,+ sealTransferredJavaScriptApplicationGraphSteps,} from "./javascriptApplicationGraph.js";import {createJavaScriptSemanticGraph,createImmutableJavaScriptSemanticGraphSteps,isValidatedImmutableJavaScriptSemanticGraph,+ sealTransferredJavaScriptSemanticGraphSteps,} from "./javascriptSemanticGraph.js";const complete = <Value>(steps: Iterator<void, Value>): Value => {@@ -265,3 +267,60 @@ describe("owned JavaScript result bindings", () => {);});});++describe("graphs transferred to their sealing steps", () => {+ it("seal the same validated graphs as the copying factories", () => {+ const input = example();+ const { graph_id: _applicationId, ...application } = input.graph;+ const { graph_id: _semanticId, ...semantic } = input.semantic_graph;+ const expectedApplication = createJavaScriptApplicationGraph(application);+ const expectedSemantic = createJavaScriptSemanticGraph(semantic);+ const sealedApplication = complete(+ sealTransferredJavaScriptApplicationGraphSteps(+ structuredClone(application),+ ),+ );+ const sealedSemantic = complete(+ sealTransferredJavaScriptSemanticGraphSteps(structuredClone(semantic)),+ );+ expect(sealedApplication).toEqual(expectedApplication);+ expect(sealedSemantic).toEqual(expectedSemantic);+ expect(+ isValidatedImmutableJavaScriptApplicationGraph(sealedApplication),+ ).toBe(true);+ expect(isValidatedImmutableJavaScriptSemanticGraph(sealedSemantic)).toBe(+ true,+ );+ });++ it("report the issues a whole-graph parse reports", () => {+ const input = example();+ const { graph_id: _applicationId, ...application } = input.graph;+ const { graph_id: _semanticId, ...semantic } = input.semantic_graph;+ const node = semantic.nodes[0];+ const applicationNode = application.nodes[0];+ if (node === undefined || applicationNode === undefined)+ throw new Error("Expected semantic and application nodes");+ const malformed = {+ ...semantic,+ nodes: [{ ...node, label: 7 }, ...semantic.nodes.slice(1)],+ };+ expect(+ issuesFrom(() =>+ complete(sealTransferredJavaScriptSemanticGraphSteps(malformed)),+ ),+ ).toEqual(issuesFrom(() => createJavaScriptSemanticGraph(malformed)));+ const stale = {+ ...application,+ nodes: [+ { ...applicationNode, node_id: `jag_node_${"f".repeat(64)}` },+ ...application.nodes.slice(1),+ ],+ };+ expect(+ issuesFrom(() =>+ complete(sealTransferredJavaScriptApplicationGraphSteps(stale)),+ ),+ ).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale)));+ });+});diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.tsindex e16aded75..ea84585f1 100644--- a/src/domain/javascript/javascriptApplicationGraph.ts+++ b/src/domain/javascript/javascriptApplicationGraph.ts@@ -1,4 +1,9 @@import { z } from "zod";+import {+ arrayContainerSchema,+ parseElementSteps,+} from "../zodCollectionSteps.js";+import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";import { canonicalJson } from "../comparisonSemantics.js";import { digestCanonicalValue } from "../canonicalDigest.js";import { uniqueSorted } from "../canonicalOrdering.js";@@ -93,11 +98,14 @@ type EdgeSemantic = Omit<ApplicationEdge, "edge_id">;const edgeId = (edge: EdgeSemantic): string =>`jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`;+/** The issue sink shared by schema refinement and stepwise checking. */+type GraphIssueContext = Pick<z.RefinementCtx, "addIssue">;+const sortedUniqueIssue = (values: readonly string[],path: PropertyKey[],label: string,- context: z.RefinementCtx,+ context: GraphIssueContext,): void => {for (let index = 1; index < values.length; index += 1) {if (@@ -116,7 +124,7 @@ const sortedUniqueIssue = (const checkEvidenceNormalization = (evidence: ApplicationGraphEvidence,path: PropertyKey[],- context: z.RefinementCtx,+ context: GraphIssueContext,): void => {sortedUniqueIssue(evidence.limitations,@@ -226,7 +234,7 @@ const observationMatchesIdentity = (const checkNode = (node: ApplicationNode,index: number,- context: z.RefinementCtx,+ context: GraphIssueContext,): void => {if (node.node_id !== nodeId(node))context.addIssue({@@ -306,8 +314,16 @@ type GraphRecord = z.infer<typeof javascriptApplicationGraphRecordSchema>;const checkGraphContent = (graph: JavaScriptApplicationGraphInput,- context: z.RefinementCtx,+ context: GraphIssueContext,): void => {+ const steps = checkGraphContentSteps(graph, context);+ while (steps.next().done !== true);+};++function* checkGraphContentSteps(+ graph: JavaScriptApplicationGraphInput,+ context: GraphIssueContext,+): Generator<void, void> {sortedUniqueIssue(graph.root_node_ids,["root_node_ids"],@@ -353,9 +369,12 @@ const checkGraphContent = (message: "Root identifier must name a graph node",path: ["root_node_ids"],});- for (const [index, node] of graph.nodes.entries())+ for (const [index, node] of graph.nodes.entries()) {+ if (index % 64 === 0) yield;checkNode(node, index, context);+ }for (const [index, edge] of graph.edges.entries()) {+ if (index % 64 === 0) yield;const { edge_id: identifier, ...semantic } = edge;if (identifier !== edgeId(semantic))context.addIssue({@@ -394,11 +413,11 @@ const checkGraphContent = (context,);}-};+}const checkGraphInvariants = (graph: GraphRecord,- context: z.RefinementCtx,+ context: GraphIssueContext,): void => {checkGraphContent(graph, context);const { graph_id: identifier, ...semantic } = graph;@@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = (export const createJavaScriptApplicationGraph = (input: unknown,): JavaScriptApplicationGraph => {- const parsed = javascriptApplicationGraphInputSchema.parse(input);- const semantic: JavaScriptApplicationGraphInput = {- ...parsed,- root_node_ids: uniqueSorted(parsed.root_node_ids),- nodes: [...parsed.nodes].sort((left, right) =>- compareUnicodeCodePoints(left.node_id, right.node_id),- ),- edges: [...parsed.edges].sort((left, right) =>- compareUnicodeCodePoints(left.edge_id, right.edge_id),- ),- coverage: {- ...parsed.coverage,- limits: normalizeLimits(parsed.coverage.limits),- },- limitations: uniqueSorted(parsed.limitations),- };+ const semantic = normalizeGraphInput(+ javascriptApplicationGraphInputSchema.parse(input),+ );const content = javascriptApplicationGraphContentSchema.parse(semantic);return {...content,@@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = (};};+const normalizeGraphInput = (+ parsed: JavaScriptApplicationGraphInput,+): JavaScriptApplicationGraphInput => ({+ ...parsed,+ root_node_ids: uniqueSorted(parsed.root_node_ids),+ nodes: [...parsed.nodes].sort((left, right) =>+ compareUnicodeCodePoints(left.node_id, right.node_id),+ ),+ edges: [...parsed.edges].sort((left, right) =>+ compareUnicodeCodePoints(left.edge_id, right.edge_id),+ ),+ coverage: {+ ...parsed.coverage,+ limits: normalizeLimits(parsed.coverage.limits),+ },+ limitations: uniqueSorted(parsed.limitations),+});+const validatedImmutableApplicationGraphs = new WeakSet<object>();/** Clone and validate synchronously, then return steps to seal the owned graph. */@@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = (): Generator<void, JavaScriptApplicationGraph> =>sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input));+const applicationGraphShape = javascriptApplicationGraphInputSchema.shape;+const applicationGraphContainerSchema =+ javascriptApplicationGraphInputSchema.extend({+ nodes: arrayContainerSchema(applicationGraphShape.nodes),+ edges: arrayContainerSchema(applicationGraphShape.edges),+ });++/**+ * Seal a graph whose input the caller transfers: the caller neither retains nor+ * mutates any part of it afterwards. Only the graph's own fields are validated+ * synchronously; nodes and edges are validated, checked, hashed and frozen in+ * steps, because doing so at once held the event loop for large applications.+ */+export const sealTransferredJavaScriptApplicationGraphSteps = (+ input: unknown,+): Generator<void, JavaScriptApplicationGraph> =>+ parseAndSealApplicationGraphSteps(+ applicationGraphContainerSchema.parse(input),+ );++function* parseAndSealApplicationGraphSteps(+ container: z.output<typeof applicationGraphContainerSchema>,+): Generator<void, JavaScriptApplicationGraph> {+ const issues: z.core.$ZodIssue[] = [];+ const parsed: JavaScriptApplicationGraphInput = {+ ...container,+ nodes: yield* parseElementSteps(+ applicationGraphShape.nodes.element,+ container.nodes,+ ["nodes"],+ issues,+ ),+ edges: yield* parseElementSteps(+ applicationGraphShape.edges.element,+ container.edges,+ ["edges"],+ issues,+ ),+ };+ if (issues.length > 0) throw new z.ZodError(issues);+ const content = normalizeGraphInput(parsed);+ let invalid = false;+ yield* checkGraphContentSteps(content, {+ addIssue: () => {+ invalid = true;+ },+ });+ // An invalid owned graph is a construction defect; report it with the exact+ // issues the content schema produces.+ if (invalid) {+ javascriptApplicationGraphContentSchema.parse(content);+ throw new TypeError("Application graph checks disagree with its schema");+ }+ return yield* sealValidatedApplicationGraphSteps({+ ...content,+ graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`,+ });+}+function* sealValidatedApplicationGraphSteps(graph: JavaScriptApplicationGraph,): Generator<void, JavaScriptApplicationGraph> {diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.tsindex 9ab2dd485..6908af2a1 100644--- a/src/domain/javascript/javascriptSemanticGraph.ts+++ b/src/domain/javascript/javascriptSemanticGraph.ts@@ -6,6 +6,10 @@ import { uniqueSorted } from "../canonicalOrdering.js";import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js";+import {+ arrayContainerSchema,+ parseElementSteps,+} from "../zodCollectionSteps.js";import type {ApplicationGraphEvidence,ApplicationGraphEvidenceContext,@@ -765,6 +769,15 @@ export const createJavaScriptSemanticGraph = (const validatedImmutableSemanticGraphs = new WeakSet<object>();+const semanticGraphShape = javaScriptSemanticGraphInputSchema.shape;+const semanticGraphContainerSchema = javaScriptSemanticGraphInputSchema.extend({+ evidence_contexts: arrayContainerSchema(semanticGraphShape.evidence_contexts),+ nodes: arrayContainerSchema(semanticGraphShape.nodes),+ relations: arrayContainerSchema(semanticGraphShape.relations),+ fingerprints: arrayContainerSchema(semanticGraphShape.fingerprints),+ unknowns: arrayContainerSchema(semanticGraphShape.unknowns),+});+/** Clone input synchronously; commit, check and seal the owned graph in steps. */export const createImmutableJavaScriptSemanticGraphSteps = (input: unknown,@@ -773,6 +786,58 @@ export const createImmutableJavaScriptSemanticGraphSteps = (normalizeGraphInput(javaScriptSemanticGraphInputSchema.parse(input)),);+/**+ * Seal a graph whose input the caller transfers: the caller neither retains nor+ * mutates any part of it afterwards. Only the graph's own fields are validated+ * synchronously; collection elements are validated and cloned in steps, because+ * a whole-graph parse of a large application held the event loop for seconds.+ */+export const sealTransferredJavaScriptSemanticGraphSteps = (+ input: unknown,+): Generator<void, JavaScriptSemanticGraph> =>+ parseAndSealSemanticGraphSteps(semanticGraphContainerSchema.parse(input));++function* parseAndSealSemanticGraphSteps(+ container: z.output<typeof semanticGraphContainerSchema>,+): Generator<void, JavaScriptSemanticGraph> {+ const issues: z.core.$ZodIssue[] = [];+ const parsed: JavaScriptSemanticGraphInput = {+ ...container,+ evidence_contexts: yield* parseElementSteps(+ semanticGraphShape.evidence_contexts.element,+ container.evidence_contexts,+ ["evidence_contexts"],+ issues,+ ),+ nodes: yield* parseElementSteps(+ semanticGraphShape.nodes.element,+ container.nodes,+ ["nodes"],+ issues,+ ),+ relations: yield* parseElementSteps(+ semanticGraphShape.relations.element,+ container.relations,+ ["relations"],+ issues,+ ),+ fingerprints: yield* parseElementSteps(+ semanticGraphShape.fingerprints.element,+ container.fingerprints,+ ["fingerprints"],+ issues,+ ),+ unknowns: yield* parseElementSteps(+ semanticGraphShape.unknowns.element,+ container.unknowns,+ ["unknowns"],+ issues,+ ),+ };+ if (issues.length > 0) throw new z.ZodError(issues);+ return yield* validateAndSealSemanticGraphSteps(normalizeGraphInput(parsed));+}+function* validateAndSealSemanticGraphSteps(semantic: JavaScriptSemanticGraphInput,): Generator<void, JavaScriptSemanticGraph> {diff --git a/src/domain/zodCollectionSteps.ts b/src/domain/zodCollectionSteps.tsnew file mode 100644index 000000000..a32e8c804--- /dev/null+++ b/src/domain/zodCollectionSteps.ts@@ -0,0 +1,33 @@+import { z } from "zod";++/**+ * An array schema's own constraints, such as a minimum length, without its+ * element schema. Pair it with {@link parseElementSteps} to validate a large+ * collection in cooperative steps.+ */+export const arrayContainerSchema = (+ array: z.ZodArray<z.ZodType>,+): z.ZodType<unknown[]> => array.clone({ ...array.def, element: z.unknown() });++/** Elements validated between yields, in the issues and paths a whole-array parse reports. */+export function* parseElementSteps<Element extends z.ZodType>(+ element: Element,+ values: readonly unknown[],+ path: readonly PropertyKey[],+ issues: z.core.$ZodIssue[],+): Generator<void, z.output<Element>[]> {+ const parsed: z.output<Element>[] = [];+ for (const [index, value] of values.entries()) {+ if (index % 64 === 0) yield;+ const result = element.safeParse(value);+ if (result.success) parsed.push(result.data);+ else+ issues.push(+ ...result.error.issues.map((issue) => ({+ ...issue,+ path: [...path, index, ...issue.path],+ })),+ );+ }+ return parsed;+}