Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): yield during per-file projection and graph construction (#1525)

Kaoru08225fdb0636b473
diff --git a/src/application/javascript/JavaScriptApplicationCancellation.test.ts b/src/application/javascript/JavaScriptApplicationCancellation.test.ts
index 1457046ef..5c2b2f005 100644
--- a/src/application/javascript/JavaScriptApplicationCancellation.test.ts
+++ b/src/application/javascript/JavaScriptApplicationCancellation.test.ts
@@ -42,6 +42,46 @@ describe("JavaScript analysis cancellation before publication", () => {
expect(phases).not.toContain("validate_javascript_application_result");
});
+ it("handles control turns and cancellation inside one large source file (#1462)", async () => {
+ const root = await createTestTempDirectory("rea-js-cancel-in-file-");
+ await writeFile(
+ join(root, "index.js"),
+ Array.from(
+ { length: 1200 },
+ (_, index) =>
+ `export function f${index}(){return {name:'f${index}',value:${index}};}`,
+ ).join("\n"),
+ );
+ const controller = new AbortController();
+ const phases: string[] = [];
+ let turns = 0;
+ // Count event-loop turns once the file starts; parsing, analysis and
+ // projection of this one file must leave room for them.
+ const tick = (): void => {
+ turns += 1;
+ if (turns === 3) controller.abort();
+ else if (!controller.signal.aborted) setImmediate(tick);
+ };
+ const result = await analyzeJavaScriptApplication(
+ { input_path: root, format: "directory" },
+ {
+ signal: controller.signal,
+ progress: {
+ async report(event) {
+ phases.push(event.phase);
+ if (event.phase === "parse_javascript_source") setImmediate(tick);
+ },
+ },
+ },
+ );
+ if (result.ok) throw new Error("Cancelled analysis must not publish");
+ expect(projectAnalysisError(result.error)).toMatchObject({
+ code: "cancelled",
+ });
+ expect(phases).toContain("parse_javascript_source");
+ expect(phases).not.toContain("build_javascript_application_graph");
+ });
+
it.each([
"parse_javascript_source",
"build_javascript_application_graph",
diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts
index 84390b873..3fac39dbb 100644
--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts
+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts
@@ -33,6 +33,7 @@ import type {
JavaScriptSourceMapOriginal,
} from "./JavaScriptArtifactAnalysisTypes.js";
import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js";
+import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js";
import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js";
interface MutableArtifactAnalysis<
@@ -56,34 +57,44 @@ interface ArtifactAnalysisContext<
readonly projectSemantics: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
- ) => SemanticIr;
+ ) => Generator<void, SemanticIr>;
}
/** Analyze every text source from the artifact without a result quota. */
export const analyzeJavaScriptArtifactFiles = (
fileSet: JavaScriptArtifactFileSet,
): JavaScriptArtifactAnalysis =>
- analyzeArtifactFiles(fileSet, (_file, ir) => ir);
+ analyzeArtifactFiles(fileSet, function* (_file, ir) {
+ // The full IR needs no projection; one step keeps the shared protocol.
+ yield;
+ return ir;
+ });
-/** Project each file's full IR immediately, retaining only later module facts. */
+/**
+ * Project each file's full IR immediately, retaining only later module facts.
+ * Parsing, analysis and each projection pass run as separate steps, so control
+ * messages and cancellation are handled within one large file as well.
+ */
export const analyzeAndProjectJavaScriptArtifactFiles = async (
fileSet: JavaScriptArtifactFileSet,
projectSemantics: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
- ) => void,
+ ) => Iterator<void, void> | void,
beforeFile?: (
file: JavaScriptArtifactFile,
completed: number,
total: number,
) => Promise<void>,
+ signal?: AbortSignal,
): Promise<JavaScriptModuleArtifactAnalysis> => {
const state = emptyArtifactAnalysis<JavaScriptModuleSemanticIr>();
- const projectFileSemantics = (
+ const projectFileSemantics = function* (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
- ): JavaScriptModuleSemanticIr => {
- projectSemantics(file, ir);
+ ): Generator<void, JavaScriptModuleSemanticIr> {
+ const projection = projectSemantics(file, ir);
+ if (projection !== undefined) while (projection.next().done !== true) yield;
const programScopes = ir.scopes.filter(({ kind }) => kind === "program");
const programScopeId = programScopes[0]?.scopeId;
const localNames = new Set(
@@ -111,7 +122,10 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (
// Let progress, cancellation and garbage collection run between files.
await setImmediate();
await beforeFile?.(file, index, fileSet.files.length);
- analyzeArtifactFile(file, context);
+ await completeJavaScriptAnalysisSteps(
+ analyzeArtifactFileSteps(file, context),
+ signal,
+ );
}
return finalizeArtifactAnalysis(state);
};
@@ -122,7 +136,10 @@ const analyzeArtifactFiles = <SemanticIr extends JavaScriptModuleSemanticIr>(
): JavaScriptArtifactAnalysis<SemanticIr> => {
const state = emptyArtifactAnalysis<SemanticIr>();
const context = { state, projectSemantics };
- for (const file of fileSet.files) analyzeArtifactFile(file, context);
+ for (const file of fileSet.files) {
+ const steps = analyzeArtifactFileSteps(file, context);
+ while (steps.next().done !== true);
+ }
return finalizeArtifactAnalysis(state);
};
@@ -148,10 +165,12 @@ const finalizeArtifactAnalysis = <
};
};
-const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
+function* analyzeArtifactFileSteps<
+ SemanticIr extends JavaScriptModuleSemanticIr,
+>(
file: JavaScriptArtifactFile,
context: ArtifactAnalysisContext<SemanticIr>,
-): void => {
+): Generator<void, void> {
const { state } = context;
addStructuredObservations(file, state);
if (file.kind === "html" && file.text.included)
@@ -162,6 +181,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
return;
}
const parsed = parseJavaScriptSource(file.text.value, file.path);
+ yield;
if (parsed === null) {
const analysis = failedJavaScriptStaticAnalysis();
state.files.push({ file, javascript: analysis, semantic: null });
@@ -175,17 +195,19 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
openReceiverFacts,
);
const staticFindings = findingCount(analysis);
+ yield;
const semantics =
analysis.parse_status === "complete" || analysis.parse_status === "partial"
? analyzeParsedJavaScriptSemantics(parsed)
: null;
+ const projected =
+ semantics === null
+ ? null
+ : yield* context.projectSemantics(file, semantics);
state.files.push({
file,
javascript: analysis,
- semantic:
- semantics === null
- ? null
- : { ir: context.projectSemantics(file, semantics) },
+ semantic: projected === null ? null : { ir: projected },
});
state.visitedNodes += analysis.visited_ast_nodes;
state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0);
@@ -194,7 +216,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
0,
);
if (analysis.parse_status === "failed") state.parseFailures += 1;
-};
+}
const addSourceMap = (
file: JavaScriptArtifactFile,
diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts
index bf46a0557..023802a79 100644
--- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts
+++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts
@@ -1,7 +1,7 @@
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import {
createJavaScriptApplicationGraph,
- createImmutableJavaScriptApplicationGraphSteps,
+ sealTransferredJavaScriptApplicationGraphSteps,
type JavaScriptApplicationGraph,
} from "../../domain/javascript/javascriptApplicationGraph.js";
import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js";
@@ -15,9 +15,9 @@ import {
addJavaScriptHtmlRoles,
addJavaScriptSourceMapOriginals,
} from "./JavaScriptArtifactGraphDocuments.js";
-import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js";
+import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js";
import {
- addJavaScriptModuleRelationships,
+ addJavaScriptModuleRelationshipsSteps,
type JavaScriptModuleRelationshipOmissions,
addJavaScriptSourceModules,
} from "./JavaScriptModuleRelationships.js";
@@ -51,24 +51,42 @@ export const buildJavaScriptArtifactGraph = (
analysis: JavaScriptModuleArtifactAnalysis,
): JavaScriptApplicationGraph =>
createJavaScriptApplicationGraph(
- buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),
+ completeSynchronously(
+ buildJavaScriptArtifactGraphInputSteps(snapshot, fileSet, analysis),
+ ),
);
-/** Build a validated application graph with cooperative immutable ownership. */
-export const buildImmutableJavaScriptArtifactGraphSteps = (
+/**
+ * Build a validated application graph with cooperative immutable ownership.
+ * Constructing the graph input also runs in steps: projecting every module
+ * relationship and finding at once held the event loop for seconds.
+ */
+export function* buildImmutableJavaScriptArtifactGraphSteps(
snapshot: ArtifactInventorySnapshot,
fileSet: JavaScriptArtifactFileSet,
analysis: JavaScriptModuleArtifactAnalysis,
-): Generator<void, JavaScriptApplicationGraph> =>
- createImmutableJavaScriptApplicationGraphSteps(
- buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),
+): Generator<void, JavaScriptApplicationGraph> {
+ const input = yield* buildJavaScriptArtifactGraphInputSteps(
+ snapshot,
+ fileSet,
+ analysis,
);
+ // The input was built here and is not retained, so it can be transferred.
+ return yield* sealTransferredJavaScriptApplicationGraphSteps(input);
+}
-const buildJavaScriptArtifactGraphInput = (
+const completeSynchronously = <Value>(steps: Iterator<void, Value>): Value => {
+ for (;;) {
+ const step = steps.next();
+ if (step.done === true) return step.value;
+ }
+};
+
+function* buildJavaScriptArtifactGraphInputSteps(
snapshot: ArtifactInventorySnapshot,
fileSet: JavaScriptArtifactFileSet,
analysis: JavaScriptModuleArtifactAnalysis,
-): unknown => {
+): Generator<void, unknown> {
const accumulator = new JavaScriptArtifactGraphAccumulator();
const root = createJavaScriptArtifactRootNode(accumulator, snapshot);
const context: JavaScriptArtifactGraphContext = {
@@ -86,15 +104,23 @@ const buildJavaScriptArtifactGraphInput = (
containerNodes: new Map([[snapshot.manifest.root_sha256, root]]),
};
addJavaScriptArtifactContainers(context);
+ yield;
addJavaScriptArtifactFiles(context);
+ yield;
const packageRoots = addJavaScriptPackageNodes(context);
+ yield;
addJavaScriptSourceModules(context);
+ yield;
const bundlerLimitations = addJavaScriptBundlerNodes(context);
- const relationshipOmissions = addJavaScriptModuleRelationships(context);
- const findingLimitations = addJavaScriptStaticFindings(context);
+ const relationshipOmissions =
+ yield* addJavaScriptModuleRelationshipsSteps(context);
+ const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context);
addElectronBoundaries(context);
+ yield;
addJavaScriptHtmlRoles(context);
+ yield;
addJavaScriptSourceMapOriginals(context);
+ yield;
const coverage = graphCoverage(context);
return {
schema: "JavaScriptApplicationGraph",
@@ -111,7 +137,7 @@ const buildJavaScriptArtifactGraphInput = (
...graphLimitations(context, coverage.status, relationshipOmissions),
],
};
-};
+}
const graphCoverage = (context: JavaScriptArtifactGraphContext) => {
const resourceLimits = semanticResourceLimits(context);
diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts
index 41f480daf..e7a002a6a 100644
--- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts
+++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts
@@ -44,11 +44,12 @@ interface ResolvedReference {
}
/** Project imports, workers, roles, endpoints, storage, and source-map links. */
-export const addJavaScriptStaticFindings = (
+export function* addJavaScriptStaticFindingsSteps(
context: JavaScriptArtifactGraphContext,
-): readonly string[] => {
+): Generator<void, readonly string[]> {
const omissions = { selfReferences: 0 };
for (const analyzed of context.analysis.files) {
+ yield;
const { file, javascript } = analyzed;
const asset = context.assetNodes.get(file.path);
if (javascript === null || asset === undefined) continue;
@@ -69,7 +70,7 @@ export const addJavaScriptStaticFindings = (
["static reference", "static references"],
"the referencing module",
);
-};
+}
const addReference = (
context: JavaScriptArtifactGraphContext,
diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts
index ab986608a..25919923c 100644
--- a/src/application/javascript/JavaScriptArtifactReconstruction.ts
+++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts
@@ -90,7 +90,7 @@ export const reconstructJavaScriptArtifact = async (
const semanticProjection = createJavaScriptSemanticGraphProjection();
const analysis = await analyzeAndProjectJavaScriptArtifactFiles(
files,
- semanticProjection.projectFile,
+ semanticProjection.projectFileSteps,
async (file, completed, total) => {
abortIfNeeded(signal);
await progress?.report({
@@ -101,6 +101,7 @@ export const reconstructJavaScriptArtifact = async (
});
abortIfNeeded(signal);
},
+ signal,
);
await checkpointJavaScriptAnalysis(signal);
abortIfNeeded(signal);
diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts
index 8e12f3beb..3784141a8 100644
--- a/src/application/javascript/JavaScriptModuleRelationships.ts
+++ b/src/application/javascript/JavaScriptModuleRelationships.ts
@@ -136,15 +136,17 @@ export interface JavaScriptModuleRelationshipOmissions {
type RelationshipOmissionCounter = { selfImports: number };
/** Compose bounded CommonJS and ESM binding relationships across artifact files. */
-export const addJavaScriptModuleRelationships = (
+export function* addJavaScriptModuleRelationshipsSteps(
context: JavaScriptArtifactGraphContext,
-): JavaScriptModuleRelationshipOmissions => {
+): Generator<void, JavaScriptModuleRelationshipOmissions> {
const omissions = { selfImports: 0 };
for (const analyzed of context.analysis.files) {
const { file, semantic } = analyzed;
const source = context.sourceModuleNodes.get(file.path);
if (semantic === null || source === undefined) continue;
for (const link of semantic.ir.moduleLinks) {
+ // One module can export thousands of bindings, each with return shapes.
+ yield;
const input = { context, file, semantic, source, link };
if (isExportLink(link)) addExportRelationship(input, omissions);
else if (link.specifier !== null)
@@ -155,7 +157,7 @@ export const addJavaScriptModuleRelationships = (
}
}
return omissions;
-};
+}
const addExportRelationship = (
input: RelationshipInput,
diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts
index 45ded5051..770930de7 100644
--- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts
+++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts
@@ -1,6 +1,6 @@
import {
createJavaScriptSemanticGraph,
- createImmutableJavaScriptSemanticGraphSteps,
+ sealTransferredJavaScriptSemanticGraphSteps,
type JavaScriptSemanticGraph,
} from "../../domain/javascript/javascriptSemanticGraph.js";
import type {
@@ -130,6 +130,11 @@ export interface JavaScriptSemanticGraphProjection {
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
) => void;
+ /** Project one file in steps between its projection passes. */
+ readonly projectFileSteps: (
+ file: JavaScriptArtifactFile,
+ ir: JavaScriptSemanticIr,
+ ) => Generator<void, void>;
readonly finish: (
rootArtifactSha256: string,
applicationGraph: BuilderInput["applicationGraph"],
@@ -146,10 +151,10 @@ export const createJavaScriptSemanticGraphProjection =
const state = emptyState({ nodes: [] });
const fingerprints: JavaScriptSemanticFingerprint[] = [];
let truncatedFiles = 0;
- const projectSource = (
+ const projectSourceSteps = function* (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
- ): void => {
+ ): Generator<void, void> {
if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) {
truncatedFiles += 1;
return;
@@ -161,7 +166,7 @@ export const createJavaScriptSemanticGraphProjection =
remainingTreeBudget,
);
state.fileNodesDropped = false;
- fingerprints.push(...projectFile(file, ir, state));
+ fingerprints.push(...(yield* projectFileSteps(file, ir, state)));
// Nodes were dropped only if the budget actually blocked creation. A file
// that exactly fills its share ends with a zero budget without dropping
// anything, so the remaining budget alone cannot decide truncation.
@@ -198,7 +203,8 @@ export const createJavaScriptSemanticGraphProjection =
evidence_contexts: state.evidenceContexts.contexts,
nodes: [...state.nodes.values()],
relations,
- fingerprints,
+ // The input is transferred to the factory; `fingerprints` is cleared below.
+ fingerprints: [...fingerprints],
unknowns,
coverage: {
status: truncatedFiles > 0 ? "partial" : "unknown",
@@ -246,7 +252,11 @@ export const createJavaScriptSemanticGraphProjection =
return graph;
};
return {
- projectFile: projectSource,
+ projectFile: (file, ir) => {
+ const steps = projectSourceSteps(file, ir);
+ while (steps.next().done !== true);
+ },
+ projectFileSteps: projectSourceSteps,
finish: (rootArtifactSha256, applicationGraph) =>
finishWith(
createJavaScriptSemanticGraph,
@@ -255,18 +265,19 @@ export const createJavaScriptSemanticGraphProjection =
),
finishImmutableSteps: (rootArtifactSha256, applicationGraph) =>
finishWith(
- createImmutableJavaScriptSemanticGraphSteps,
+ sealTransferredJavaScriptSemanticGraphSteps,
rootArtifactSha256,
applicationGraph,
),
};
};
-const projectFile = (
+function* projectFileSteps(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
state: BuilderState,
-): JavaScriptSemanticFingerprint[] => {
+): Generator<void, JavaScriptSemanticFingerprint[]> {
+ yield;
const moduleNode = retainNode(state, file, {
kind: "module",
roleKey: "module",
@@ -328,6 +339,7 @@ const projectFile = (
callableOwnerAt,
callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),
};
+ yield;
projectDefinitionsAndReferences(definitions);
const context: FileContext = {
...definitions,
@@ -335,26 +347,20 @@ const projectFile = (
definitions.referenceNodes,
),
};
- projectSemanticValues(context);
- projectSemanticObjects(context);
- projectCalls(context);
- projectSemanticPromises(context);
- projectSemanticEvents(context);
- projectSemanticTimers(context);
- projectSemanticChildProcesses(context);
- projectSemanticConfiguration(context);
- projectSemanticRequests(context);
- projectSemanticBoundaries(context);
- projectSemanticResources(context);
- projectSemanticClosureCaptures(context);
- projectSemanticFrontiers(context);
+ // Each pass completes before the next starts, so yielding between passes
+ // keeps projection order and results unchanged.
+ for (const project of FILE_PROJECTION_PASSES) {
+ yield;
+ project(context);
+ }
+ yield;
return projectSemanticFunctionFingerprints(
file,
ir,
callableNodes,
state.evidenceContexts,
);
-};
+}
const createReturnSiteNodes = (
file: JavaScriptArtifactFile,
@@ -543,3 +549,20 @@ const createArgumentNode = (
: `argument ${String(argument.index)}`,
functionNodeId: callNode.function_node_id,
});
+
+/** Per-file projection passes, run in this order with a yield between each. */
+const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [
+ projectSemanticValues,
+ projectSemanticObjects,
+ projectCalls,
+ projectSemanticPromises,
+ projectSemanticEvents,
+ projectSemanticTimers,
+ projectSemanticChildProcesses,
+ projectSemanticConfiguration,
+ projectSemanticRequests,
+ projectSemanticBoundaries,
+ projectSemanticResources,
+ projectSemanticClosureCaptures,
+ projectSemanticFrontiers,
+];
diff --git a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts
index c96ccece7..a460de077 100644
--- a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts
+++ b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts
@@ -11,11 +11,13 @@ import {
createJavaScriptApplicationGraph,
createImmutableJavaScriptApplicationGraphSteps,
isValidatedImmutableJavaScriptApplicationGraph,
+ sealTransferredJavaScriptApplicationGraphSteps,
} from "./javascriptApplicationGraph.js";
import {
createJavaScriptSemanticGraph,
createImmutableJavaScriptSemanticGraphSteps,
isValidatedImmutableJavaScriptSemanticGraph,
+ sealTransferredJavaScriptSemanticGraphSteps,
} from "./javascriptSemanticGraph.js";
const complete = <Value>(steps: Iterator<void, Value>): Value => {
@@ -265,3 +267,60 @@ describe("owned JavaScript result bindings", () => {
);
});
});
+
+describe("graphs transferred to their sealing steps", () => {
+ it("seal the same validated graphs as the copying factories", () => {
+ const input = example();
+ const { graph_id: _applicationId, ...application } = input.graph;
+ const { graph_id: _semanticId, ...semantic } = input.semantic_graph;
+ const expectedApplication = createJavaScriptApplicationGraph(application);
+ const expectedSemantic = createJavaScriptSemanticGraph(semantic);
+ const sealedApplication = complete(
+ sealTransferredJavaScriptApplicationGraphSteps(
+ structuredClone(application),
+ ),
+ );
+ const sealedSemantic = complete(
+ sealTransferredJavaScriptSemanticGraphSteps(structuredClone(semantic)),
+ );
+ expect(sealedApplication).toEqual(expectedApplication);
+ expect(sealedSemantic).toEqual(expectedSemantic);
+ expect(
+ isValidatedImmutableJavaScriptApplicationGraph(sealedApplication),
+ ).toBe(true);
+ expect(isValidatedImmutableJavaScriptSemanticGraph(sealedSemantic)).toBe(
+ true,
+ );
+ });
+
+ it("report the issues a whole-graph parse reports", () => {
+ const input = example();
+ const { graph_id: _applicationId, ...application } = input.graph;
+ const { graph_id: _semanticId, ...semantic } = input.semantic_graph;
+ const node = semantic.nodes[0];
+ const applicationNode = application.nodes[0];
+ if (node === undefined || applicationNode === undefined)
+ throw new Error("Expected semantic and application nodes");
+ const malformed = {
+ ...semantic,
+ nodes: [{ ...node, label: 7 }, ...semantic.nodes.slice(1)],
+ };
+ expect(
+ issuesFrom(() =>
+ complete(sealTransferredJavaScriptSemanticGraphSteps(malformed)),
+ ),
+ ).toEqual(issuesFrom(() => createJavaScriptSemanticGraph(malformed)));
+ const stale = {
+ ...application,
+ nodes: [
+ { ...applicationNode, node_id: `jag_node_${"f".repeat(64)}` },
+ ...application.nodes.slice(1),
+ ],
+ };
+ expect(
+ issuesFrom(() =>
+ complete(sealTransferredJavaScriptApplicationGraphSteps(stale)),
+ ),
+ ).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale)));
+ });
+});
diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.ts
index e16aded75..ea84585f1 100644
--- a/src/domain/javascript/javascriptApplicationGraph.ts
+++ b/src/domain/javascript/javascriptApplicationGraph.ts
@@ -1,4 +1,9 @@
import { z } from "zod";
+import {
+ arrayContainerSchema,
+ parseElementSteps,
+} from "../zodCollectionSteps.js";
+import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
import { canonicalJson } from "../comparisonSemantics.js";
import { digestCanonicalValue } from "../canonicalDigest.js";
import { uniqueSorted } from "../canonicalOrdering.js";
@@ -93,11 +98,14 @@ type EdgeSemantic = Omit<ApplicationEdge, "edge_id">;
const edgeId = (edge: EdgeSemantic): string =>
`jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`;
+/** The issue sink shared by schema refinement and stepwise checking. */
+type GraphIssueContext = Pick<z.RefinementCtx, "addIssue">;
+
const sortedUniqueIssue = (
values: readonly string[],
path: PropertyKey[],
label: string,
- context: z.RefinementCtx,
+ context: GraphIssueContext,
): void => {
for (let index = 1; index < values.length; index += 1) {
if (
@@ -116,7 +124,7 @@ const sortedUniqueIssue = (
const checkEvidenceNormalization = (
evidence: ApplicationGraphEvidence,
path: PropertyKey[],
- context: z.RefinementCtx,
+ context: GraphIssueContext,
): void => {
sortedUniqueIssue(
evidence.limitations,
@@ -226,7 +234,7 @@ const observationMatchesIdentity = (
const checkNode = (
node: ApplicationNode,
index: number,
- context: z.RefinementCtx,
+ context: GraphIssueContext,
): void => {
if (node.node_id !== nodeId(node))
context.addIssue({
@@ -306,8 +314,16 @@ type GraphRecord = z.infer<typeof javascriptApplicationGraphRecordSchema>;
const checkGraphContent = (
graph: JavaScriptApplicationGraphInput,
- context: z.RefinementCtx,
+ context: GraphIssueContext,
): void => {
+ const steps = checkGraphContentSteps(graph, context);
+ while (steps.next().done !== true);
+};
+
+function* checkGraphContentSteps(
+ graph: JavaScriptApplicationGraphInput,
+ context: GraphIssueContext,
+): Generator<void, void> {
sortedUniqueIssue(
graph.root_node_ids,
["root_node_ids"],
@@ -353,9 +369,12 @@ const checkGraphContent = (
message: "Root identifier must name a graph node",
path: ["root_node_ids"],
});
- for (const [index, node] of graph.nodes.entries())
+ for (const [index, node] of graph.nodes.entries()) {
+ if (index % 64 === 0) yield;
checkNode(node, index, context);
+ }
for (const [index, edge] of graph.edges.entries()) {
+ if (index % 64 === 0) yield;
const { edge_id: identifier, ...semantic } = edge;
if (identifier !== edgeId(semantic))
context.addIssue({
@@ -394,11 +413,11 @@ const checkGraphContent = (
context,
);
}
-};
+}
const checkGraphInvariants = (
graph: GraphRecord,
- context: z.RefinementCtx,
+ context: GraphIssueContext,
): void => {
checkGraphContent(graph, context);
const { graph_id: identifier, ...semantic } = graph;
@@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = (
export const createJavaScriptApplicationGraph = (
input: unknown,
): JavaScriptApplicationGraph => {
- const parsed = javascriptApplicationGraphInputSchema.parse(input);
- const semantic: JavaScriptApplicationGraphInput = {
- ...parsed,
- root_node_ids: uniqueSorted(parsed.root_node_ids),
- nodes: [...parsed.nodes].sort((left, right) =>
- compareUnicodeCodePoints(left.node_id, right.node_id),
- ),
- edges: [...parsed.edges].sort((left, right) =>
- compareUnicodeCodePoints(left.edge_id, right.edge_id),
- ),
- coverage: {
- ...parsed.coverage,
- limits: normalizeLimits(parsed.coverage.limits),
- },
- limitations: uniqueSorted(parsed.limitations),
- };
+ const semantic = normalizeGraphInput(
+ javascriptApplicationGraphInputSchema.parse(input),
+ );
const content = javascriptApplicationGraphContentSchema.parse(semantic);
return {
...content,
@@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = (
};
};
+const normalizeGraphInput = (
+ parsed: JavaScriptApplicationGraphInput,
+): JavaScriptApplicationGraphInput => ({
+ ...parsed,
+ root_node_ids: uniqueSorted(parsed.root_node_ids),
+ nodes: [...parsed.nodes].sort((left, right) =>
+ compareUnicodeCodePoints(left.node_id, right.node_id),
+ ),
+ edges: [...parsed.edges].sort((left, right) =>
+ compareUnicodeCodePoints(left.edge_id, right.edge_id),
+ ),
+ coverage: {
+ ...parsed.coverage,
+ limits: normalizeLimits(parsed.coverage.limits),
+ },
+ limitations: uniqueSorted(parsed.limitations),
+});
+
const validatedImmutableApplicationGraphs = new WeakSet<object>();
/** Clone and validate synchronously, then return steps to seal the owned graph. */
@@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = (
): Generator<void, JavaScriptApplicationGraph> =>
sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input));
+const applicationGraphShape = javascriptApplicationGraphInputSchema.shape;
+const applicationGraphContainerSchema =
+ javascriptApplicationGraphInputSchema.extend({
+ nodes: arrayContainerSchema(applicationGraphShape.nodes),
+ edges: arrayContainerSchema(applicationGraphShape.edges),
+ });
+
+/**
+ * Seal a graph whose input the caller transfers: the caller neither retains nor
+ * mutates any part of it afterwards. Only the graph's own fields are validated
+ * synchronously; nodes and edges are validated, checked, hashed and frozen in
+ * steps, because doing so at once held the event loop for large applications.
+ */
+export const sealTransferredJavaScriptApplicationGraphSteps = (
+ input: unknown,
+): Generator<void, JavaScriptApplicationGraph> =>
+ parseAndSealApplicationGraphSteps(
+ applicationGraphContainerSchema.parse(input),
+ );
+
+function* parseAndSealApplicationGraphSteps(
+ container: z.output<typeof applicationGraphContainerSchema>,
+): Generator<void, JavaScriptApplicationGraph> {
+ const issues: z.core.$ZodIssue[] = [];
+ const parsed: JavaScriptApplicationGraphInput = {
+ ...container,
+ nodes: yield* parseElementSteps(
+ applicationGraphShape.nodes.element,
+ container.nodes,
+ ["nodes"],
+ issues,
+ ),
+ edges: yield* parseElementSteps(
+ applicationGraphShape.edges.element,
+ container.edges,
+ ["edges"],
+ issues,
+ ),
+ };
+ if (issues.length > 0) throw new z.ZodError(issues);
+ const content = normalizeGraphInput(parsed);
+ let invalid = false;
+ yield* checkGraphContentSteps(content, {
+ addIssue: () => {
+ invalid = true;
+ },
+ });
+ // An invalid owned graph is a construction defect; report it with the exact
+ // issues the content schema produces.
+ if (invalid) {
+ javascriptApplicationGraphContentSchema.parse(content);
+ throw new TypeError("Application graph checks disagree with its schema");
+ }
+ return yield* sealValidatedApplicationGraphSteps({
+ ...content,
+ graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`,
+ });
+}
+
function* sealValidatedApplicationGraphSteps(
graph: JavaScriptApplicationGraph,
): Generator<void, JavaScriptApplicationGraph> {
diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts
index 9ab2dd485..6908af2a1 100644
--- a/src/domain/javascript/javascriptSemanticGraph.ts
+++ b/src/domain/javascript/javascriptSemanticGraph.ts
@@ -6,6 +6,10 @@ import { uniqueSorted } from "../canonicalOrdering.js";
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js";
+import {
+ arrayContainerSchema,
+ parseElementSteps,
+} from "../zodCollectionSteps.js";
import type {
ApplicationGraphEvidence,
ApplicationGraphEvidenceContext,
@@ -765,6 +769,15 @@ export const createJavaScriptSemanticGraph = (
const validatedImmutableSemanticGraphs = new WeakSet<object>();
+const semanticGraphShape = javaScriptSemanticGraphInputSchema.shape;
+const semanticGraphContainerSchema = javaScriptSemanticGraphInputSchema.extend({
+ evidence_contexts: arrayContainerSchema(semanticGraphShape.evidence_contexts),
+ nodes: arrayContainerSchema(semanticGraphShape.nodes),
+ relations: arrayContainerSchema(semanticGraphShape.relations),
+ fingerprints: arrayContainerSchema(semanticGraphShape.fingerprints),
+ unknowns: arrayContainerSchema(semanticGraphShape.unknowns),
+});
+
/** Clone input synchronously; commit, check and seal the owned graph in steps. */
export const createImmutableJavaScriptSemanticGraphSteps = (
input: unknown,
@@ -773,6 +786,58 @@ export const createImmutableJavaScriptSemanticGraphSteps = (
normalizeGraphInput(javaScriptSemanticGraphInputSchema.parse(input)),
);
+/**
+ * Seal a graph whose input the caller transfers: the caller neither retains nor
+ * mutates any part of it afterwards. Only the graph's own fields are validated
+ * synchronously; collection elements are validated and cloned in steps, because
+ * a whole-graph parse of a large application held the event loop for seconds.
+ */
+export const sealTransferredJavaScriptSemanticGraphSteps = (
+ input: unknown,
+): Generator<void, JavaScriptSemanticGraph> =>
+ parseAndSealSemanticGraphSteps(semanticGraphContainerSchema.parse(input));
+
+function* parseAndSealSemanticGraphSteps(
+ container: z.output<typeof semanticGraphContainerSchema>,
+): Generator<void, JavaScriptSemanticGraph> {
+ const issues: z.core.$ZodIssue[] = [];
+ const parsed: JavaScriptSemanticGraphInput = {
+ ...container,
+ evidence_contexts: yield* parseElementSteps(
+ semanticGraphShape.evidence_contexts.element,
+ container.evidence_contexts,
+ ["evidence_contexts"],
+ issues,
+ ),
+ nodes: yield* parseElementSteps(
+ semanticGraphShape.nodes.element,
+ container.nodes,
+ ["nodes"],
+ issues,
+ ),
+ relations: yield* parseElementSteps(
+ semanticGraphShape.relations.element,
+ container.relations,
+ ["relations"],
+ issues,
+ ),
+ fingerprints: yield* parseElementSteps(
+ semanticGraphShape.fingerprints.element,
+ container.fingerprints,
+ ["fingerprints"],
+ issues,
+ ),
+ unknowns: yield* parseElementSteps(
+ semanticGraphShape.unknowns.element,
+ container.unknowns,
+ ["unknowns"],
+ issues,
+ ),
+ };
+ if (issues.length > 0) throw new z.ZodError(issues);
+ return yield* validateAndSealSemanticGraphSteps(normalizeGraphInput(parsed));
+}
+
function* validateAndSealSemanticGraphSteps(
semantic: JavaScriptSemanticGraphInput,
): Generator<void, JavaScriptSemanticGraph> {
diff --git a/src/domain/zodCollectionSteps.ts b/src/domain/zodCollectionSteps.ts
new file mode 100644
index 000000000..a32e8c804
--- /dev/null
+++ b/src/domain/zodCollectionSteps.ts
@@ -0,0 +1,33 @@
+import { z } from "zod";
+
+/**
+ * An array schema's own constraints, such as a minimum length, without its
+ * element schema. Pair it with {@link parseElementSteps} to validate a large
+ * collection in cooperative steps.
+ */
+export const arrayContainerSchema = (
+ array: z.ZodArray<z.ZodType>,
+): z.ZodType<unknown[]> => array.clone({ ...array.def, element: z.unknown() });
+
+/** Elements validated between yields, in the issues and paths a whole-array parse reports. */
+export function* parseElementSteps<Element extends z.ZodType>(
+ element: Element,
+ values: readonly unknown[],
+ path: readonly PropertyKey[],
+ issues: z.core.$ZodIssue[],
+): Generator<void, z.output<Element>[]> {
+ const parsed: z.output<Element>[] = [];
+ for (const [index, value] of values.entries()) {
+ if (index % 64 === 0) yield;
+ const result = element.safeParse(value);
+ if (result.success) parsed.push(result.data);
+ else
+ issues.push(
+ ...result.error.issues.map((issue) => ({
+ ...issue,
+ path: [...path, index, ...issue.path],
+ })),
+ );
+ }
+ return parsed;
+}