Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): preserve graphs within parent capacity (#1921)

Ysopwnable0485cd3bce29
diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts
index 4113ec8db..c02ddaa3e 100644
--- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts
+++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts
@@ -59,6 +59,7 @@ import {
const OPERATION = "analyze_javascript_application";
const PROVIDER = "javascript-static-analysis";
const MIB = 1024 * 1024;
+const STATIC_CHECKPOINT_RETENTION_EXPANSION = 6;
const pendingCleanup = new Set<JavaScriptAnalysisWorker>();
/** The worker runs on this executable; Bun accepts but ignores V8 heap flags. */
const WORKER_HEAP_FLAGS_ENFORCED =
@@ -80,7 +81,6 @@ export class JavaScriptAnalysisWorker {
#estimatedRetainedBytes = 0;
#applicationProjectionFailure: AnalysisResourceConstraintError | undefined;
#parentCapacityExhausted = false;
- #retainedStaticBytes = 0;
#sourcesRemaining: number | undefined;
readonly #resultBudgetBytes: number;
/** Null when the runtime enforces the V8 heap flags instead. */
@@ -326,7 +326,13 @@ export class JavaScriptAnalysisWorker {
{
...this.#limits(),
parent_result_budget_bytes: this.#resultBudgetBytes,
+ parent_available_result_budget_bytes: remaining,
module_transfer_bytes: descriptor.module_bytes,
+ module_retention_estimate_bytes: moduleEstimate,
+ },
+ {
+ remediationAction:
+ "Use the retained static facts or analyze fewer source files. This limit belongs to the parent result budget; increasing the worker heap alone does not increase it.",
},
),
});
@@ -358,9 +364,16 @@ export class JavaScriptAnalysisWorker {
{
...this.#limits(),
parent_result_budget_bytes: this.#resultBudgetBytes,
+ parent_available_result_budget_bytes: remaining,
+ module_retention_estimate_bytes: moduleEstimate,
+ graph_retention_estimate_bytes: graphEstimate,
graph_transfer_bytes: descriptor.graph_bytes,
graph_records: descriptor.graph_records,
},
+ {
+ remediationAction:
+ "Use the retained static and module facts or analyze fewer source files. This limit belongs to the parent result budget; increasing the worker heap alone does not increase it.",
+ },
),
});
break;
@@ -582,16 +595,16 @@ export class JavaScriptAnalysisWorker {
const value: unknown = JSON.parse(text);
const facts = javaScriptWorkerStaticAnalysisSchema.parse(value);
const remaining = this.#remainingTotalResultBudget();
- const retainedBytes = bytes * 6;
+ const retainedBytes = bytes * STATIC_CHECKPOINT_RETENTION_EXPANSION;
this.#parentCapacityExhausted = retainedBytes > remaining;
this.#estimatedRetainedBytes += retainedBytes;
- this.#retainedStaticBytes += retainedBytes;
const expandedBytes = estimateJavaScriptStaticApplicationProjection(
facts,
bytes,
sourcePath,
);
- if (expandedBytes <= this.#remainingResultBudget())
+ const availableResultBudget = this.#remainingResultBudget();
+ if (expandedBytes <= availableResultBudget)
this.#estimatedRetainedBytes += expandedBytes;
else
this.#applicationProjectionFailure = new AnalysisResourceConstraintError(
@@ -600,8 +613,13 @@ export class JavaScriptAnalysisWorker {
`Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`,
{
parent_result_budget_bytes: this.#resultBudgetBytes,
+ parent_available_result_budget_bytes: availableResultBudget,
application_projection_required_bytes: expandedBytes,
},
+ {
+ remediationAction:
+ "Read the complete static facts on the file node or analyze fewer source files. Expanded application relationships exceed the parent result budget; increasing the worker heap alone does not increase it.",
+ },
);
return facts;
}
@@ -638,16 +656,17 @@ export class JavaScriptAnalysisWorker {
);
}
#remainingResultBudget(): number {
- // Future sources need their compact observations before current modules
- // consume capacity with expanded relationships. A final or sole source
- // can use the unused reserve as well.
+ // One sequential worker produces one next checkpoint at a time. Reserve
+ // its maximum protocol size at the same retained expansion used on receipt;
+ // earlier checkpoints are already charged to estimatedRetainedBytes.
+ // Reserving every unexamined source's maximum would reject useful results
+ // merely because many small files remain. Actual retained growth and live
+ // heap capacity are checked again for every checkpoint and transfer.
+ const checkpointReserve =
+ DEFAULTS.checkpointBytes * STATIC_CHECKPOINT_RETENTION_EXPANSION;
const staticReserve =
this.#sourcesRemaining !== undefined && this.#sourcesRemaining > 0
- ? Math.max(
- 0,
- Math.floor((this.#resultBudgetBytes * 4) / 5) -
- this.#retainedStaticBytes,
- )
+ ? checkpointReserve
: 0;
return Math.max(0, this.#remainingTotalResultBudget() - staticReserve);
}
diff --git a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts
new file mode 100644
index 000000000..32d113781
--- /dev/null
+++ b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts
@@ -0,0 +1,132 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { expect } from "vitest";
+import { z } from "zod";
+
+import { parseEvidence } from "../../../src/domain/evidence.js";
+import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+cliTest.skipIf(process.versions.bun !== undefined).for([
+ { laterSources: 1, heapMb: 640, methodsCount: 400, retainGraph: true },
+ { laterSources: 10, heapMb: 640, methodsCount: 400, retainGraph: true },
+ // Even an otherwise empty parent cannot admit this graph after reserving
+ // transient decoding and the next checkpoint, independent of GC timing.
+ { laterSources: 1, heapMb: 448, methodsCount: 600, retainGraph: false },
+])(
+ "preserves evidence with $laterSources later checkpoints and a $heapMb MiB parent heap",
+ { timeout: 90_000 },
+ async ({ laterSources, heapMb, methodsCount, retainGraph }, { cli }) => {
+ const root = await createTestTempDirectory("rea-js-checkpoint-reserve-");
+ const source = Array.from(
+ { length: methodsCount },
+ (_, index) =>
+ `export function method_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`,
+ ).join("\n");
+ const files: readonly (readonly [string, string])[] = [
+ ["a-large.mjs", source],
+ ...Array.from(
+ { length: laterSources },
+ (_, index) =>
+ [
+ `z-tiny-${String(index)}.mjs`,
+ `export const tiny${String(index)} = 1;\n`,
+ ] as const,
+ ),
+ ];
+ for (const [name, text] of files) await writeFile(join(root, name), text);
+ const response = await cli.run({
+ arguments: ["analyze", root, "--format", "json"],
+ environment: {
+ NODE_OPTIONS: `--max-old-space-size=${String(heapMb)} --max-semi-space-size=8`,
+ },
+ timeoutMs: 60_000,
+ });
+ expect(response.exitCode).toBe(0);
+ const result = javascriptApplicationAnalysisResultSchema.parse(
+ parseEvidence(response.json).normalized_result,
+ );
+ expect(result.statistics.parsed_javascript_files).toBe(laterSources + 1);
+ expect(result.statistics.parse_failures).toBe(0);
+ const failures = result.graph.nodes.flatMap(({ observations }) =>
+ observations.flatMap(({ properties }) =>
+ properties.analysis_failure === undefined
+ ? []
+ : [properties.analysis_failure],
+ ),
+ );
+ const methods = result.semantic_graph.nodes.filter(
+ ({ kind, identity }) =>
+ kind === "function" && identity.module_path === "a-large.mjs",
+ );
+ if (retainGraph)
+ expect(methods.map(({ label }) => label).sort()).toEqual(
+ Array.from(
+ { length: methodsCount },
+ (_, index) => `method_${String(index)}`,
+ ).sort(),
+ );
+ else {
+ expect(methods).toHaveLength(0);
+ expect([
+ result.graph.coverage.status,
+ result.semantic_graph.coverage.status,
+ ]).toContain("partial");
+ }
+ expect(
+ result.semantic_graph.nodes.some(
+ ({ identity, label }) =>
+ identity.module_path === `z-tiny-${String(laterSources - 1)}.mjs` &&
+ label === `tiny${String(laterSources - 1)}`,
+ ),
+ ).toBe(true);
+ for (const [name, text] of files) {
+ const digest = createHash("sha256").update(text).digest("hex");
+ const node = result.graph.nodes.find(
+ ({ kind, identity }) =>
+ kind === "javascript-asset" &&
+ identity.strategy === "content-digest" &&
+ identity.sha256 === digest,
+ );
+ expect(node?.observations).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ properties: expect.objectContaining({
+ path: name,
+ parse_status: "complete",
+ }),
+ }),
+ ]),
+ );
+ if (retainGraph) expect(failures).toEqual([]);
+ expect(await readFile(join(root, name), "utf8")).toBe(text);
+ if (!retainGraph && name === "a-large.mjs") {
+ const failure = z
+ .object({
+ code: z.literal("resource_constraint"),
+ remediation: z.object({ action: z.string() }),
+ details: z.object({
+ reported_limits: z.object({
+ parent_available_result_budget_bytes: z.number(),
+ module_retention_estimate_bytes: z.number(),
+ graph_retention_estimate_bytes: z.number(),
+ }),
+ }),
+ })
+ .parse(failures[0]);
+ const limits = failure.details.reported_limits;
+ expect(
+ limits.module_retention_estimate_bytes +
+ limits.graph_retention_estimate_bytes,
+ ).toBeGreaterThan(limits.parent_available_result_budget_bytes);
+ expect(failure.remediation.action).toContain("parent result budget");
+ expect(failure.remediation.action).toContain(
+ "retained static and module facts",
+ );
+ }
+ }
+ expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu);
+ },
+);