Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): preserve graphs within parent capacity (#1921)
Ysopwnable0485cd3bce29
diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.tsindex 4113ec8db..c02ddaa3e 100644--- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts+++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts@@ -59,6 +59,7 @@ import {const OPERATION = "analyze_javascript_application";const PROVIDER = "javascript-static-analysis";const MIB = 1024 * 1024;+const STATIC_CHECKPOINT_RETENTION_EXPANSION = 6;const pendingCleanup = new Set<JavaScriptAnalysisWorker>();/** The worker runs on this executable; Bun accepts but ignores V8 heap flags. */const WORKER_HEAP_FLAGS_ENFORCED =@@ -80,7 +81,6 @@ export class JavaScriptAnalysisWorker {#estimatedRetainedBytes = 0;#applicationProjectionFailure: AnalysisResourceConstraintError | undefined;#parentCapacityExhausted = false;- #retainedStaticBytes = 0;#sourcesRemaining: number | undefined;readonly #resultBudgetBytes: number;/** Null when the runtime enforces the V8 heap flags instead. */@@ -326,7 +326,13 @@ export class JavaScriptAnalysisWorker {{...this.#limits(),parent_result_budget_bytes: this.#resultBudgetBytes,+ parent_available_result_budget_bytes: remaining,module_transfer_bytes: descriptor.module_bytes,+ module_retention_estimate_bytes: moduleEstimate,+ },+ {+ remediationAction:+ "Use the retained static facts or analyze fewer source files. This limit belongs to the parent result budget; increasing the worker heap alone does not increase it.",},),});@@ -358,9 +364,16 @@ export class JavaScriptAnalysisWorker {{...this.#limits(),parent_result_budget_bytes: this.#resultBudgetBytes,+ parent_available_result_budget_bytes: remaining,+ module_retention_estimate_bytes: moduleEstimate,+ graph_retention_estimate_bytes: graphEstimate,graph_transfer_bytes: descriptor.graph_bytes,graph_records: descriptor.graph_records,},+ {+ remediationAction:+ "Use the retained static and module facts or analyze fewer source files. This limit belongs to the parent result budget; increasing the worker heap alone does not increase it.",+ },),});break;@@ -582,16 +595,16 @@ export class JavaScriptAnalysisWorker {const value: unknown = JSON.parse(text);const facts = javaScriptWorkerStaticAnalysisSchema.parse(value);const remaining = this.#remainingTotalResultBudget();- const retainedBytes = bytes * 6;+ const retainedBytes = bytes * STATIC_CHECKPOINT_RETENTION_EXPANSION;this.#parentCapacityExhausted = retainedBytes > remaining;this.#estimatedRetainedBytes += retainedBytes;- this.#retainedStaticBytes += retainedBytes;const expandedBytes = estimateJavaScriptStaticApplicationProjection(facts,bytes,sourcePath,);- if (expandedBytes <= this.#remainingResultBudget())+ const availableResultBudget = this.#remainingResultBudget();+ if (expandedBytes <= availableResultBudget)this.#estimatedRetainedBytes += expandedBytes;elsethis.#applicationProjectionFailure = new AnalysisResourceConstraintError(@@ -600,8 +613,13 @@ export class JavaScriptAnalysisWorker {`Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`,{parent_result_budget_bytes: this.#resultBudgetBytes,+ parent_available_result_budget_bytes: availableResultBudget,application_projection_required_bytes: expandedBytes,},+ {+ remediationAction:+ "Read the complete static facts on the file node or analyze fewer source files. Expanded application relationships exceed the parent result budget; increasing the worker heap alone does not increase it.",+ },);return facts;}@@ -638,16 +656,17 @@ export class JavaScriptAnalysisWorker {);}#remainingResultBudget(): number {- // Future sources need their compact observations before current modules- // consume capacity with expanded relationships. A final or sole source- // can use the unused reserve as well.+ // One sequential worker produces one next checkpoint at a time. Reserve+ // its maximum protocol size at the same retained expansion used on receipt;+ // earlier checkpoints are already charged to estimatedRetainedBytes.+ // Reserving every unexamined source's maximum would reject useful results+ // merely because many small files remain. Actual retained growth and live+ // heap capacity are checked again for every checkpoint and transfer.+ const checkpointReserve =+ DEFAULTS.checkpointBytes * STATIC_CHECKPOINT_RETENTION_EXPANSION;const staticReserve =this.#sourcesRemaining !== undefined && this.#sourcesRemaining > 0- ? Math.max(- 0,- Math.floor((this.#resultBudgetBytes * 4) / 5) -- this.#retainedStaticBytes,- )+ ? checkpointReserve: 0;return Math.max(0, this.#remainingTotalResultBudget() - staticReserve);}diff --git a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.tsnew file mode 100644index 000000000..32d113781--- /dev/null+++ b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts@@ -0,0 +1,132 @@+import { createHash } from "node:crypto";+import { readFile, writeFile } from "node:fs/promises";+import { join } from "node:path";+import { expect } from "vitest";+import { z } from "zod";++import { parseEvidence } from "../../../src/domain/evidence.js";+import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++cliTest.skipIf(process.versions.bun !== undefined).for([+ { laterSources: 1, heapMb: 640, methodsCount: 400, retainGraph: true },+ { laterSources: 10, heapMb: 640, methodsCount: 400, retainGraph: true },+ // Even an otherwise empty parent cannot admit this graph after reserving+ // transient decoding and the next checkpoint, independent of GC timing.+ { laterSources: 1, heapMb: 448, methodsCount: 600, retainGraph: false },+])(+ "preserves evidence with $laterSources later checkpoints and a $heapMb MiB parent heap",+ { timeout: 90_000 },+ async ({ laterSources, heapMb, methodsCount, retainGraph }, { cli }) => {+ const root = await createTestTempDirectory("rea-js-checkpoint-reserve-");+ const source = Array.from(+ { length: methodsCount },+ (_, index) =>+ `export function method_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`,+ ).join("\n");+ const files: readonly (readonly [string, string])[] = [+ ["a-large.mjs", source],+ ...Array.from(+ { length: laterSources },+ (_, index) =>+ [+ `z-tiny-${String(index)}.mjs`,+ `export const tiny${String(index)} = 1;\n`,+ ] as const,+ ),+ ];+ for (const [name, text] of files) await writeFile(join(root, name), text);+ const response = await cli.run({+ arguments: ["analyze", root, "--format", "json"],+ environment: {+ NODE_OPTIONS: `--max-old-space-size=${String(heapMb)} --max-semi-space-size=8`,+ },+ timeoutMs: 60_000,+ });+ expect(response.exitCode).toBe(0);+ const result = javascriptApplicationAnalysisResultSchema.parse(+ parseEvidence(response.json).normalized_result,+ );+ expect(result.statistics.parsed_javascript_files).toBe(laterSources + 1);+ expect(result.statistics.parse_failures).toBe(0);+ const failures = result.graph.nodes.flatMap(({ observations }) =>+ observations.flatMap(({ properties }) =>+ properties.analysis_failure === undefined+ ? []+ : [properties.analysis_failure],+ ),+ );+ const methods = result.semantic_graph.nodes.filter(+ ({ kind, identity }) =>+ kind === "function" && identity.module_path === "a-large.mjs",+ );+ if (retainGraph)+ expect(methods.map(({ label }) => label).sort()).toEqual(+ Array.from(+ { length: methodsCount },+ (_, index) => `method_${String(index)}`,+ ).sort(),+ );+ else {+ expect(methods).toHaveLength(0);+ expect([+ result.graph.coverage.status,+ result.semantic_graph.coverage.status,+ ]).toContain("partial");+ }+ expect(+ result.semantic_graph.nodes.some(+ ({ identity, label }) =>+ identity.module_path === `z-tiny-${String(laterSources - 1)}.mjs` &&+ label === `tiny${String(laterSources - 1)}`,+ ),+ ).toBe(true);+ for (const [name, text] of files) {+ const digest = createHash("sha256").update(text).digest("hex");+ const node = result.graph.nodes.find(+ ({ kind, identity }) =>+ kind === "javascript-asset" &&+ identity.strategy === "content-digest" &&+ identity.sha256 === digest,+ );+ expect(node?.observations).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ properties: expect.objectContaining({+ path: name,+ parse_status: "complete",+ }),+ }),+ ]),+ );+ if (retainGraph) expect(failures).toEqual([]);+ expect(await readFile(join(root, name), "utf8")).toBe(text);+ if (!retainGraph && name === "a-large.mjs") {+ const failure = z+ .object({+ code: z.literal("resource_constraint"),+ remediation: z.object({ action: z.string() }),+ details: z.object({+ reported_limits: z.object({+ parent_available_result_budget_bytes: z.number(),+ module_retention_estimate_bytes: z.number(),+ graph_retention_estimate_bytes: z.number(),+ }),+ }),+ })+ .parse(failures[0]);+ const limits = failure.details.reported_limits;+ expect(+ limits.module_retention_estimate_bytes ++ limits.graph_retention_estimate_bytes,+ ).toBeGreaterThan(limits.parent_available_result_budget_bytes);+ expect(failure.remediation.action).toContain("parent result budget");+ expect(failure.remediation.action).toContain(+ "retained static and module facts",+ );+ }+ }+ expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu);+ },+);